{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:33:55.630530Z",
     "start_time": "2020-12-11T02:33:55.624219Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "pd.set_option(\"display.max_rows\", 120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:25:57.785690Z",
     "start_time": "2020-12-11T02:25:57.590566Z"
    }
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_json(path_or_buf=\"./train.json\", lines=True)\n",
    "df_dev = pd.read_json(path_or_buf=\"./dev.json\", lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:25:57.792718Z",
     "start_time": "2020-12-11T02:25:57.788136Z"
    }
   },
   "outputs": [],
   "source": [
    "df = df_train.append(df_dev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# label占比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:38:32.503551Z",
     "start_time": "2020-12-11T02:38:32.474325Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label_des</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>工具</th>\n",
       "      <td>2368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>休闲益智</th>\n",
       "      <td>1144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>亲子儿童</th>\n",
       "      <td>629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>中小学</th>\n",
       "      <td>466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>借贷</th>\n",
       "      <td>454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>动作类</th>\n",
       "      <td>443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>经营养成</th>\n",
       "      <td>336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>电商</th>\n",
       "      <td>331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>棋牌中心</th>\n",
       "      <td>317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>小说</th>\n",
       "      <td>307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>视频</th>\n",
       "      <td>289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>新闻</th>\n",
       "      <td>276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>理财</th>\n",
       "      <td>261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>论坛圈子</th>\n",
       "      <td>253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>射击游戏</th>\n",
       "      <td>222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>策略</th>\n",
       "      <td>208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>体育竞技</th>\n",
       "      <td>205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>购物咨询</th>\n",
       "      <td>191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>辅助工具</th>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>英语</th>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>音乐</th>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>直播</th>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>约会社交</th>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>办公</th>\n",
       "      <td>167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>同城服务</th>\n",
       "      <td>166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>仙侠</th>\n",
       "      <td>166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>公共交通</th>\n",
       "      <td>154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>职考</th>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>政务</th>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>彩票</th>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>经营</th>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>银行</th>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>股票</th>\n",
       "      <td>133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>其他</th>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>摄影修图</th>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>运动健身</th>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>魔幻</th>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>电子产品</th>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>医疗服务</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>成人教育</th>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>薅羊毛</th>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>社区服务</th>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>相机</th>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>影像剪辑</th>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>即时通讯</th>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>高等教育</th>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>支付</th>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>行车辅助</th>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>兼职</th>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>卡牌</th>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>漫画</th>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>短视频</th>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>求职</th>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>旅游资讯</th>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>养生保健</th>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>艺术</th>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>驾校</th>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>快递物流</th>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>菜谱</th>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>地图导航</th>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>社交工具</th>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>教辅</th>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>汽车咨询</th>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>记账</th>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>日常养车</th>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>问诊挂号</th>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>体育咨讯</th>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>母婴</th>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>飞行空战</th>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>免费WIFI</th>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>语言(非英语)</th>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>女性</th>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>美颜</th>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>婚恋社交</th>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>汽车交易</th>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MOBA</th>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>绘画</th>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>电台</th>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>保险</th>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>买房</th>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>装修家居</th>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>综合预定</th>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>笔记</th>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>民航</th>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>社区超市</th>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>美妆美业</th>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>二手</th>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>违章</th>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>工作社交</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>微博博客</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>杂志</th>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>打车</th>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>团购</th>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>收款</th>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>租车</th>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>技术</th>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>租房</th>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>视频教育</th>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>搞笑</th>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>问答交流</th>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>铁路</th>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>生活社交</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>电影票务</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>百科</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>家政</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>公务员</th>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>K歌</th>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>日程管理</th>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>影视娱乐</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>情侣社交</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>外卖</th>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>酒店</th>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>成人</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>减肥瘦身</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>行程管理</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>餐饮店</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>出国</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>婚庆</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>民宿短租</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         label_des\n",
       "工具            2368\n",
       "休闲益智          1144\n",
       "亲子儿童           629\n",
       "中小学            466\n",
       "借贷             454\n",
       "动作类            443\n",
       "经营养成           336\n",
       "电商             331\n",
       "棋牌中心           317\n",
       "小说             307\n",
       "视频             289\n",
       "新闻             276\n",
       "理财             261\n",
       "论坛圈子           253\n",
       "射击游戏           222\n",
       "策略             208\n",
       "体育竞技           205\n",
       "购物咨询           191\n",
       "辅助工具           190\n",
       "英语             189\n",
       "音乐             179\n",
       "直播             176\n",
       "约会社交           171\n",
       "办公             167\n",
       "同城服务           166\n",
       "仙侠             166\n",
       "公共交通           154\n",
       "职考             144\n",
       "政务             140\n",
       "彩票             139\n",
       "经营             136\n",
       "银行             136\n",
       "股票             133\n",
       "其他             129\n",
       "摄影修图           126\n",
       "运动健身           122\n",
       "魔幻             119\n",
       "电子产品           105\n",
       "医疗服务            99\n",
       "成人教育            98\n",
       "薅羊毛             97\n",
       "社区服务            96\n",
       "相机              87\n",
       "影像剪辑            83\n",
       "即时通讯            75\n",
       "高等教育            72\n",
       "支付              62\n",
       "行车辅助            61\n",
       "兼职              61\n",
       "卡牌              58\n",
       "漫画              56\n",
       "短视频             54\n",
       "求职              51\n",
       "旅游资讯            51\n",
       "养生保健            51\n",
       "艺术              49\n",
       "驾校              48\n",
       "快递物流            48\n",
       "菜谱              48\n",
       "地图导航            47\n",
       "社交工具            47\n",
       "教辅              46\n",
       "汽车咨询            45\n",
       "记账              45\n",
       "日常养车            44\n",
       "问诊挂号            43\n",
       "体育咨讯            42\n",
       "母婴              40\n",
       "飞行空战            39\n",
       "免费WIFI          38\n",
       "语言(非英语)         38\n",
       "女性              38\n",
       "美颜              37\n",
       "婚恋社交            36\n",
       "汽车交易            36\n",
       "MOBA            36\n",
       "绘画              35\n",
       "电台              35\n",
       "保险              34\n",
       "买房              33\n",
       "装修家居            32\n",
       "综合预定            32\n",
       "笔记              31\n",
       "民航              30\n",
       "社区超市            30\n",
       "美妆美业            29\n",
       "二手              28\n",
       "违章              28\n",
       "工作社交            26\n",
       "微博博客            26\n",
       "杂志              26\n",
       "打车              25\n",
       "团购              25\n",
       "收款              25\n",
       "租车              24\n",
       "技术              24\n",
       "租房              24\n",
       "视频教育            23\n",
       "搞笑              22\n",
       "问答交流            22\n",
       "铁路              22\n",
       "生活社交            21\n",
       "电影票务            21\n",
       "百科              21\n",
       "家政              20\n",
       "公务员             19\n",
       "K歌              19\n",
       "日程管理            17\n",
       "影视娱乐            16\n",
       "情侣社交            16\n",
       "外卖              14\n",
       "酒店              13\n",
       "成人              12\n",
       "减肥瘦身            12\n",
       "行程管理            10\n",
       "餐饮店              7\n",
       "出国               6\n",
       "婚庆               5\n",
       "民宿短租             3"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['label_des'].value_counts(normalize=False).to_frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# label长度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:25:57.832367Z",
     "start_time": "2020-12-11T02:25:57.822533Z"
    }
   },
   "outputs": [],
   "source": [
    "df['len'] = df[\"sentence\"].apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:32:42.974871Z",
     "start_time": "2020-12-11T02:32:42.575796Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABJQAAAI/CAYAAAAhoYNSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnGklEQVR4nO3df6yf130f9ven5OQ1KQIlMJPK+rFLr3Q2usgchZC1FTGCZZklMTXXBh7oppOrBiBYSG33CzU1Dx2QwgCzDNviVhEnpNoi1AkrRHVChOpkN13T/VHFomzXCeWopWTGYsTGqr3K3RRIo/zZH9/H2tfXl/c+h7zk5Y/XC/jifp/znPM85yF4QPLNc85T3R0AAAAAmOuPbHUHAAAAALi6CJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYsn2rO7AZ3v72t/fKyspWdwMAAADgmvHss8/+y+7esda5ayJQWllZyYkTJ7a6GwAAAADXjKr6vfOds+QNAAAAgCGzAqWququqnq+qU1V1aI3zVVUfn85/oapuH2j7X1dVV9Xbl8oenOo/X1Xvv9CHAwAAAGDzbRgoVdW2JA8luTvJ7iQfqqrdq6rdnWTX9DmQ5OE5bavq1iQ/luTLS2W7k+xP8u4kdyX5+ek6AAAAAFwB5sxQuiPJqe5+sbvfSHI0yb5VdfYleawXnk5yY1XdNKPt/5TkryXpVdc62t2vd/eXkpyargMAAADAFWBOoHRzkpeWjs9MZXPqnLdtVX0gye939z+9gPsBAAAAsEXmvOWt1ijrmXXWLK+q70jy0ST/8QXeL1V1IIvldbntttvWaAIAAADApTBnhtKZJLcuHd+S5OWZdc5X/m8n2Znkn1bV6an8s1X1x2feL939SHfv6e49O3bsmPEYAAAAAGyGOYHSM0l2VdXOqrohiw2zj62qcyzJvdPb3u5M8mp3nz1f2+7+7e7+3u5e6e6VLEKk27v7X0zX2l9Vb6uqnVls9P2ZzXhYAAAAAC7ehkveuvtcVT2Q5Kkk25I82t0nq+rgdP5IkieT3JPFBtqvJblvvbYb3O9kVT2e5Lkk55Lc391vXugDAgAAALC5qvvbtie66uzZs6dPnDix1d0AAAAAuGZU1bPdvWetc3OWvAEAAADAWwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAzZvtUd4MKsHDo+q97pw3svcU8AAACA640ZSgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwJBZgVJV3VVVz1fVqao6tMb5qqqPT+e/UFW3b9S2qv7GVPfzVfWpqnrHVL5SVX84lX++qo5sxoMCAAAAsDk2DJSqaluSh5LcnWR3kg9V1e5V1e5Osmv6HEjy8Iy2P9vdP9Dd70ny60n++tL1Xuju90yfgxf6cAAAAABsvjkzlO5Icqq7X+zuN5IcTbJvVZ19SR7rhaeT3FhVN63Xtru/vtT+O5P0RT4LAAAAAJfBnEDp5iQvLR2fmcrm1Fm3bVV9rKpeSvKT+dYZSjur6nNV9ZtV9cMz+ggAAADAZTInUKo1ylbPJjpfnXXbdvdHu/vWJJ9I8sBUfDbJbd39g0n+yyS/VFXf9W2dqjpQVSeq6sQrr7wy4zEAAAAA2AxzAqUzSW5dOr4lycsz68xpmyS/lOQnkqS7X+/ur07fn03yQpJ3rW7Q3Y90957u3rNjx44ZjwEAAADAZpgTKD2TZFdV7ayqG5LsT3JsVZ1jSe6d3vZ2Z5JXu/vsem2ratdS+w8k+d2pfMe0mXeq6p1ZbPT94gU/IQAAAACbavtGFbr7XFU9kOSpJNuSPNrdJ6vq4HT+SJInk9yT5FSS15Lct17b6dKHq+r7k3wjye8l+ebb3N6X5Ker6lySN5Mc7O6vbcrTAgAAAHDRNgyUkqS7n8wiNFouO7L0vZPcP7ftVP4T56n/RJIn5vQLAAAAgMtvzpI3AAAAAHiLQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCGzAqWququqnq+qU1V1aI3zVVUfn85/oapu36htVf2Nqe7nq+pTVfWOpXMPTvWfr6r3X+xDAgAAALB5NgyUqmpbkoeS3J1kd5IPVdXuVdXuTrJr+hxI8vCMtj/b3T/Q3e9J8utJ/vrUZneS/UneneSuJD8/XQcAAACAK8CcGUp3JDnV3S929xtJjibZt6rOviSP9cLTSW6sqpvWa9vdX19q/51JeulaR7v79e7+UpJT03UAAAAAuALMCZRuTvLS0vGZqWxOnXXbVtXHquqlJD+ZaYbSzPsBAAAAsEXmBEq1RlnPrLNu2+7+aHffmuQTSR4YuF+q6kBVnaiqE6+88sqaHQcAAABg880JlM4kuXXp+JYkL8+sM6dtkvxSkp8YuF+6+5Hu3tPde3bs2DHjMQAAAADYDHMCpWeS7KqqnVV1QxYbZh9bVedYknunt73dmeTV7j67Xtuq2rXU/gNJfnfpWvur6m1VtTOLjb4/c4HPBwAAAMAm275Rhe4+V1UPJHkqybYkj3b3yao6OJ0/kuTJJPdksYH2a0nuW6/tdOnDVfX9Sb6R5PeSfPN6J6vq8STPJTmX5P7ufnOzHhgAAACAi7NhoJQk3f1kFqHRctmRpe+d5P65bafyn1ij+jfPfSzJx+b0DQAAAIDLa86SNwAAAAB4i0AJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYsn2rOwDXo5VDx2fVO3147yXuCQAAAIwzQwkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhmzf6g7A1WDl0PFZ9U4f3nuJewIAAABbzwwlAAAAAIaYoQRXMDOjAAAAuBKZoQQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMGT7VncAuHxWDh2fVe/04b2XuCcAAABczWYFSlV1V5KfS7ItyS909+FV52s6f0+S15L8he7+7Hptq+pnk/zpJG8keSHJfd39r6pqJckXkzw/Xf7p7j54MQ8JjJkbPCXCJwAAgOvRhkveqmpbkoeS3J1kd5IPVdXuVdXuTrJr+hxI8vCMtp9O8ie7+weS/LMkDy5d74Xufs/0ESYBAAAAXEHm7KF0R5JT3f1id7+R5GiSfavq7EvyWC88neTGqrppvbbd/anuPje1fzrJLZvwPAAAAABcYnOWvN2c5KWl4zNJ3jujzs0z2ybJX0zyd5eOd1bV55J8Pcl/293/54x+Alcw+zcBAABcO+YESrVGWc+ss2HbqvpoknNJPjEVnU1yW3d/tap+KMmvVtW7u/vrq9odyGJ5XW677bYNHwIAAACAzTEnUDqT5Nal41uSvDyzzg3rta2qDyf58SQ/2t2dJN39epLXp+/PVtULSd6V5MTyDbv7kSSPJMmePXtWB1xwXRnZRBsAAAAu1pw9lJ5JsquqdlbVDUn2Jzm2qs6xJPfWwp1JXu3us+u1nd7+9pEkH+ju1755oaraMW3mnap6ZxYbfb94UU8JAAAAwKbZcIZSd5+rqgeSPJVkW5JHu/tkVR2czh9J8mSSe5KcSvJakvvWaztd+m8leVuST1dVkjw9vdHtfUl+uqrOJXkzycHu/tpmPTB8k1k9AAAAcGHmLHlLdz+ZRWi0XHZk6XsnuX9u26n8T5yn/hNJnpjTLwAAAAAuvzlL3gAAAADgLbNmKAHzWEYHAADA9cAMJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYMj2re4AwLKVQ8c39XqnD+/d1OsBAABghhIAAAAAg8xQAi7KZs8oAgAA4MpnhhIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAzZvtUdALgSrBw6Pqve6cN7L3FPAAAArnxmKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMGT7VncA4FJaOXT8ir/36cN7L3FPAAAANpcZSgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADJkVKFXVXVX1fFWdqqpDa5yvqvr4dP4LVXX7Rm2r6mer6nen+p+sqhuXzj041X++qt5/kc8IAAAAwCbavlGFqtqW5KEkP5bkTJJnqupYdz+3VO3uJLumz3uTPJzkvRu0/XSSB7v7XFX9TJIHk3ykqnYn2Z/k3UnekeQfVNW7uvvNzXlkrnUrh45vdRe4hvn9BQAAMG+G0h1JTnX3i939RpKjSfatqrMvyWO98HSSG6vqpvXadvenuvvc1P7pJLcsXetod7/e3V9Kcmq6DgAAAABXgDmB0s1JXlo6PjOVzakzp22S/MUkf3/gfgAAAABskTmBUq1R1jPrbNi2qj6a5FySTwzcL1V1oKpOVNWJV155ZY0mAAAAAFwKcwKlM0luXTq+JcnLM+us27aqPpzkx5P8ZHd/MzSac7909yPdvae79+zYsWPGYwAAAACwGeYESs8k2VVVO6vqhiw2zD62qs6xJPdOb3u7M8mr3X12vbZVdVeSjyT5QHe/tupa+6vqbVW1M4uNvj9zEc8IAAAAwCba8C1v01vYHkjyVJJtSR7t7pNVdXA6fyTJk0nuyWID7deS3Lde2+nSfyvJ25J8uqqS5OnuPjhd+/Ekz2WxFO5+b3gDAAAAuHJsGCglSXc/mUVotFx2ZOl7J7l/btup/E+sc7+PJfnYnL4BAAAAcHnNWfIGAAAAAG8RKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwZPtWdwCArbFy6PiseqcP773EPbl419KzAADA1cAMJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAITblBrjGzN2gGgAA4EKZoQQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQb3kD2GJz38p2+vDeS9wTAACAecxQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGDI9q3uAACcz8qh41vdBQAAYA1mKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADDEW964anjbEwAAAFwZzFACAAAAYIhACQAAAIAhAiUAAAAAhthDCeAqcS3tI3YtPQsAAFyPzFACAAAAYIgZSgAAm2DuzLvTh/de4p4AAFx6ZigBAAAAMESgBAAAAMAQgRIAAAAAQ+yhBABsmZE3/tl7CADgymGGEgAAAABDzFACYF3eXAUAAKw2a4ZSVd1VVc9X1amqOrTG+aqqj0/nv1BVt2/Utqo+WFUnq+obVbVnqXylqv6wqj4/fY5c7EMCAAAAsHk2nKFUVduSPJTkx5KcSfJMVR3r7ueWqt2dZNf0eW+Sh5O8d4O2v5Pkzyb5X9a47Qvd/Z4LfioAAAAALpk5M5TuSHKqu1/s7jeSHE2yb1WdfUke64Wnk9xYVTet17a7v9jdz2/akwAAAABwWcwJlG5O8tLS8ZmpbE6dOW3XsrOqPldVv1lVPzyjPgAAAACXyZxNuWuNsp5ZZ07b1c4mua27v1pVP5TkV6vq3d399W+5YdWBJAeS5LbbbtvgkgAAAABsljkzlM4kuXXp+JYkL8+sM6ftt+ju17v7q9P3Z5O8kORda9R7pLv3dPeeHTt2zHgMAAAAADbDnBlKzyTZVVU7k/x+kv1J/tyqOseSPFBVR7PYlPvV7j5bVa/MaPstqmpHkq9195tV9c4sNvp+ceShAIDr18qh45t6vdOH927q9QAArgUbBkrdfa6qHkjyVJJtSR7t7pNVdXA6fyTJk0nuSXIqyWtJ7luvbZJU1Z9J8jeT7EhyvKo+393vT/K+JD9dVeeSvJnkYHd/bTMfGgA2w9zgQiABAMC1Zs4MpXT3k1mERstlR5a+d5L757adyj+Z5JNrlD+R5Ik5/QIAAADg8puzhxIAAAAAvEWgBAAAAMAQgRIAAAAAQ2btoQQAG9nsN2vBtcpm7gDAtcAMJQAAAACGCJQAAAAAGCJQAgAAAGCIPZQAANZhfzAAgG9nhhIAAAAAQwRKAAAAAAyx5A0AriBeKQ8AwNVAoATAdUNYc/nYdwgA4NomUAIAZhMUAQCQ2EMJAAAAgEECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIZs3+oOAADMsXLo+FZ3AQCAiRlKAAAAAAwRKAEAAAAwxJI3ALjELNUCAOBaY4YSAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQm3IDwCo20QYAgPUJlAAAIRoAAEMseQMAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIdu3ugMAwLiVQ8dn1Tt9eO8l7gkAANcjM5QAAAAAGCJQAgAAAGCIJW8AAFegucsaE0sbAYDLzwwlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCHbt7oDAABcHiuHjs+qd/rw3kvcEwDgameGEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEJtyAwBc5eZutg0AsFnMUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGDIrECpqu6qquer6lRVHVrjfFXVx6fzX6iq2zdqW1UfrKqTVfWNqtqz6noPTvWfr6r3X8wDAgAAALC5NgyUqmpbkoeS3J1kd5IPVdXuVdXuTrJr+hxI8vCMtr+T5M8m+cer7rc7yf4k705yV5Kfn64DAAAAwBVgzgylO5Kc6u4Xu/uNJEeT7FtVZ1+Sx3rh6SQ3VtVN67Xt7i929/Nr3G9fkqPd/Xp3fynJqek6AAAAAFwBts+oc3OSl5aOzyR574w6N89su9b9nl7jWlyAlUPHZ9U7fXjvJe4JAFth7p8DAAAwYs4MpVqjrGfWmdP2Qu6XqjpQVSeq6sQrr7yywSUBAAAA2CxzAqUzSW5dOr4lycsz68xpeyH3S3c/0t17unvPjh07NrgkAAAAAJtlTqD0TJJdVbWzqm7IYsPsY6vqHEty7/S2tzuTvNrdZ2e2Xe1Ykv1V9baq2pnFRt+fGXgmAAAAAC6hDfdQ6u5zVfVAkqeSbEvyaHefrKqD0/kjSZ5Mck8WG2i/luS+9domSVX9mSR/M8mOJMer6vPd/f7p2o8neS7JuST3d/ebm/rUAAAAAFywOZtyp7ufzCI0Wi47svS9k9w/t+1U/skknzxPm48l+dicvgEAAABwec1Z8gYAAAAAbxEoAQAAADBk1pI3AACuHyuHjs+qd/rw3kvcEwDgSmWGEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMCQ7VvdAVg5dHyruwAAAAAMECgBAHBB5v6n0OnDey9xTwCAy82SNwAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIZ4yxtJ5r+lJfGmFgAAALjemaEEAAAAwBCBEgAAAABDBEoAAAAADBEoAQAAADBEoAQAAADAEG95AwDgkpr7NllvkgWAq4cZSgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDtm91B7h2rRw6vtVdAAAAAC4BM5QAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIdu3ugMAAJAkK4eOz6p3+vDeS9wTAGAjZigBAAAAMMQMJYbN/d9DAAAA4NpkhhIAAAAAQwRKAAAAAAwRKAEAAAAwRKAEAAAAwBCBEgAAAABDBEoAAAAADJkVKFXVXVX1fFWdqqpDa5yvqvr4dP4LVXX7Rm2r6nuq6tNV9c+nn989la9U1R9W1eenz5HNeFAAAAAANseGgVJVbUvyUJK7k+xO8qGq2r2q2t1Jdk2fA0kentH2UJLf6O5dSX5jOv6mF7r7PdPn4IU+HAAAAACbb/uMOnckOdXdLyZJVR1Nsi/Jc0t19iV5rLs7ydNVdWNV3ZRkZZ22+5L8yNT+F5P8oyQfucjnAQDgGrdy6PjsuqcP772EPQGA69ecJW83J3lp6fjMVDanznptv6+7zybJ9PN7l+rtrKrPVdVvVtUPz+gjAAAAAJfJnBlKtUZZz6wzp+1qZ5Pc1t1fraofSvKrVfXu7v76t9yw6kAWy+ty2223bXBJAAAAADbLnBlKZ5LcunR8S5KXZ9ZZr+0fTMviMv38SpJ09+vd/dXp+7NJXkjyrtWd6u5HuntPd+/ZsWPHjMcAAAAAYDPMCZSeSbKrqnZW1Q1J9ic5tqrOsST3Tm97uzPJq9MytvXaHkvy4en7h5P8WpJU1Y5pM+9U1Tuz2Oj7xQt+QgAAAAA21YZL3rr7XFU9kOSpJNuSPNrdJ6vq4HT+SJInk9yT5FSS15Lct17b6dKHkzxeVT+V5MtJPjiVvy/JT1fVuSRvJjnY3V/blKcFAAAA4KLN2UMp3f1kFqHRctmRpe+d5P65bafyryb50TXKn0jyxJx+AQAAAHD5zVnyBgAAAABvESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAwZPtWdwAAAC6VlUPHN/V6pw/v3dTrAcDVygwlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAhAiUAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIdu3ugMAAHCtWTl0fFa904f3XuKeAMClYYYSAAAAAEMESgAAAAAMESgBAAAAMMQeSgAAsEW2cq8l+zwBcDHMUAIAAABgiEAJAAAAgCGWvAEAwBVu7vK0xBI1AC4PgRIAAMw0EuwAwLXMkjcAAAAAhgiUAAAAABgiUAIAAABgiEAJAAAAgCECJQAAAACGCJQAAAAAGCJQAgAAAGCIQAkAAACAIQIlAAAAAIYIlAAAAAAYIlACAAAAYIhACQAAAIAh27e6AwAAwOZZOXR8q7sAwHXADCUAAAAAhpihBAAAXBfmzt46fXjvJe4JwNVPoAQAAJzXVi2hE+oAXNkseQMAAABgiEAJAAAAgCECJQAAAACG2EMJAAC44thAG+DKZoYSAAAAAEPMUAIAAK5aW/UWOoDrnUAJAACA68pIEGlZJazNkjcAAAAAhpihdIUxZRcAAK4O1+PG4Zv975Vr6dcGrjcCJQAAgCWbHZps1X8aC2tgc1kq+a0ESgAAAGyJzZ7ltZUrPq7HGWtc3wRKAAAAXNFsDXL5XIpZOMK2a5NACQAA4BpkeQ7XKgHVlUGgBAAAANc4s7zYbLMCpaq6K8nPJdmW5Be6+/Cq8zWdvyfJa0n+Qnd/dr22VfU9Sf5ukpUkp5P8p939f03nHkzyU0neTPJXuvupi3pKAAAAYFNdKyHVtfIcl1t19/oVqrYl+WdJfizJmSTPJPlQdz+3VOeeJH85i0DpvUl+rrvfu17bqvrvk3ytuw9X1aEk393dH6mq3Ul+OckdSd6R5B8keVd3v3m+Pu7Zs6dPnDhxYb8CVxi/kQEAANjs5Vr+rXl5XSvL7arq2e7es9a5OTOU7khyqrtfnC52NMm+JM8t1dmX5LFepFNPV9WNVXVTFrOPztd2X5Ifmdr/YpJ/lOQjU/nR7n49yZeq6tTUh38y94EBAADgeiAoYqv8kRl1bk7y0tLxmalsTp312n5fd59Nkunn9w7cDwAAAIAtMmeGUq1Rtnqd3PnqzGl7IfdLVR1IcmA6/L+r6vkNrns1eHuSf7nVnYCrgLEC8xgrMI+xAvMYK5dR/cxW94CLUT9zzYyXf+t8J+YESmeS3Lp0fEuSl2fWuWGdtn9QVTd199lpedxXBu6X7n4kySMz+n/VqKoT51ubCPz/jBWYx1iBeYwVmMdYgfmuh/EyZ8nbM0l2VdXOqrohyf4kx1bVOZbk3lq4M8mr0zK29doeS/Lh6fuHk/zaUvn+qnpbVe1MsivJZy7w+QAAAADYZBvOUOruc1X1QJKnkmxL8mh3n6yqg9P5I0mezOINb6eSvJbkvvXaTpc+nOTxqvqpJF9O8sGpzcmqejyLjbvPJbl/vTe8AQAAAHB51eLFbFwJqurAtJQPWIexAvMYKzCPsQLzGCsw3/UwXgRKAAAAAAyZs4cSAAAAALxFoHQFqKq7qur5qjpVVYe2uj+wlarq1qr6P6rqi1V1sqr+6lT+PVX16ar659PP715q8+A0fp6vqvdvXe/h8quqbVX1uar69enYWIE1VNWNVfUrVfW7058x/77xAt+uqv6L6e9gv1NVv1xV/6axAklVPVpVX6mq31kqGx4bVfVDVfXb07mPV1Vd7mfZLAKlLVZV25I8lOTuJLuTfKiqdm9tr2BLnUvyX3X3v5vkziT3T2PiUJLf6O5dSX5jOs50bn+Sdye5K8nPT+MKrhd/NckXl46NFVjbzyX537v730ny72UxbowXWFJVNyf5K0n2dPefzOLFSvtjrECS/G9Z/D5fdiFj4+EkB7J4o/2uNa551RAobb07kpzq7he7+40kR5Ps2+I+wZbp7rPd/dnp+7/O4i/8N2cxLn5xqvaLSf6T6fu+JEe7+/Xu/lIWb5u847J2GrZIVd2SZG+SX1gqNlZglar6riTvS/K3k6S73+jufxXjBdayPckfrartSb4jycsxViDd/Y+TfG1V8dDYqKqbknxXd/+TXmxo/dhSm6uOQGnr3ZzkpaXjM1MZXPeqaiXJDyb5rSTf191nk0XolOR7p2rGENez/znJX0vyjaUyYwW+3TuTvJLkf52WiP5CVX1njBf4Ft39+0n+hyRfTnI2yavd/akYK3A+o2Pj5un76vKrkkBp6621XtKr97juVdUfS/JEkv+8u7++XtU1yowhrnlV9eNJvtLdz85tskaZscL1YnuS25M83N0/mOT/ybQs4TyMF65L0/4v+5LsTPKOJN9ZVX9+vSZrlBkrcP6xcU2NGYHS1juT5Nal41uymFYK162q+jeyCJM+0d1/byr+g2mKaKafX5nKjSGuV38qyQeq6nQWy6X/w6r6OzFWYC1nkpzp7t+ajn8li4DJeIFv9R8l+VJ3v9Ld/2+Sv5fkP4ixAuczOjbOTN9Xl1+VBEpb75kku6pqZ1XdkMXGXce2uE+wZaa3HPztJF/s7v9x6dSxJB+evn84ya8tle+vqrdV1c4sNrb7zOXqL2yV7n6wu2/p7pUs/uz4h93952OswLfp7n+R5KWq+v6p6EeTPBfjBVb7cpI7q+o7pr+T/WgW+1kaK7C2obExLYv711V15zTG7l1qc9XZvtUduN5197mqeiDJU1m8ReHR7j65xd2CrfSnkvxnSX67qj4/lf03SQ4nebyqfiqLv+x8MEm6+2RVPZ7FPwzOJbm/u9+87L2GK4exAmv7y0k+Mf0H3otJ7sviP1eNF5h0929V1a8k+WwWv/c/l+SRJH8sxgrXuar65SQ/kuTtVXUmyX+XC/t711/K4o1xfzTJ358+V6VabCwOAAAAAPNY8gYAAADAEIESAAAAAEMESgAAAAAMESgBAAAAMESgBAAAAMAQgRIAAAAAQwRKAAAAAAwRKAEAAAAw5P8Dr44TqTnhrPEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "_ = plt.hist(df['len'], bins=100, range=(1, 1000), density=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:34:56.432062Z",
     "start_time": "2020-12-11T02:34:56.408189Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>label_des</th>\n",
       "      <th>sentence</th>\n",
       "      <th>len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11</td>\n",
       "      <td>薅羊毛</td>\n",
       "      <td>活动新用户领5元现金即日起成功注册酷划的新用户，填写邀请码80008即可领取5元现金奖励哦。...</td>\n",
       "      <td>492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>95</td>\n",
       "      <td>借贷</td>\n",
       "      <td>产品简介小腰包APP是友戏互联为年轻白领人群打造的一款互联网消费金融应用，以用户日常消费行为...</td>\n",
       "      <td>436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>74</td>\n",
       "      <td>违章</td>\n",
       "      <td>6000万车主都在用的查违章洗车养车加油车险优惠一站式用车服务平台，河南交警系统直连特色功能...</td>\n",
       "      <td>566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70</td>\n",
       "      <td>工具</td>\n",
       "      <td>这不仅是一个一站式的服务平台，在使用魅族手机过程中遇到什么可以通过这个应用来寻求解决，还是您...</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>58</td>\n",
       "      <td>高等教育</td>\n",
       "      <td>\"2019普通话学习、教师、公务员、播音主持、学拼音、配音员做普通话水平测试的优选软件1.普...</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2594</th>\n",
       "      <td>106</td>\n",
       "      <td>电商</td>\n",
       "      <td>熊猫购物www.163nn.com海量精品优惠，随时购物省钱每比淘宝天猫直接买，省80，折扣...</td>\n",
       "      <td>325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2595</th>\n",
       "      <td>17</td>\n",
       "      <td>休闲益智</td>\n",
       "      <td>酷狗音乐核心功能1.全新界面，打造属于你的首页首页入口提供图标自定义，要什么功能由你决定；推...</td>\n",
       "      <td>721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2596</th>\n",
       "      <td>73</td>\n",
       "      <td>驾校</td>\n",
       "      <td>JJ学车是一款基于驾校集约车、交友、娱乐为一体的便捷交互式学车APP。可以在线报名约车、更有...</td>\n",
       "      <td>471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2597</th>\n",
       "      <td>53</td>\n",
       "      <td>中小学</td>\n",
       "      <td>\"快快查系列产品高中古诗收录了初中课本中全部的诗词，并配以详尽的注释，赏析，翻译，帮助学生快...</td>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2598</th>\n",
       "      <td>38</td>\n",
       "      <td>教辅</td>\n",
       "      <td>《中华上下五千年》由300多个生动的故事串联中华泱泱史实，全面而真实地反映了各时期的特点，大...</td>\n",
       "      <td>325</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14732 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label label_des                                           sentence  len\n",
       "0        11       薅羊毛  活动新用户领5元现金即日起成功注册酷划的新用户，填写邀请码80008即可领取5元现金奖励哦。...  492\n",
       "1        95        借贷  产品简介小腰包APP是友戏互联为年轻白领人群打造的一款互联网消费金融应用，以用户日常消费行为...  436\n",
       "2        74        违章  6000万车主都在用的查违章洗车养车加油车险优惠一站式用车服务平台，河南交警系统直连特色功能...  566\n",
       "3        70        工具  这不仅是一个一站式的服务平台，在使用魅族手机过程中遇到什么可以通过这个应用来寻求解决，还是您...   86\n",
       "4        58      高等教育  \"2019普通话学习、教师、公务员、播音主持、学拼音、配音员做普通话水平测试的优选软件1.普...  260\n",
       "...     ...       ...                                                ...  ...\n",
       "2594    106        电商  熊猫购物www.163nn.com海量精品优惠，随时购物省钱每比淘宝天猫直接买，省80，折扣...  325\n",
       "2595     17      休闲益智  酷狗音乐核心功能1.全新界面，打造属于你的首页首页入口提供图标自定义，要什么功能由你决定；推...  721\n",
       "2596     73        驾校  JJ学车是一款基于驾校集约车、交友、娱乐为一体的便捷交互式学车APP。可以在线报名约车、更有...  471\n",
       "2597     53       中小学  \"快快查系列产品高中古诗收录了初中课本中全部的诗词，并配以详尽的注释，赏析，翻译，帮助学生快...  225\n",
       "2598     38        教辅  《中华上下五千年》由300多个生动的故事串联中华泱泱史实，全面而真实地反映了各时期的特点，大...  325\n",
       "\n",
       "[14732 rows x 4 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# split train&dev*test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:39:30.015509Z",
     "start_time": "2020-12-11T02:39:30.000488Z"
    }
   },
   "outputs": [],
   "source": [
    "df_train, df_dev_test = train_test_split(df, test_size=0.3)\n",
    "df_dev, de_test = train_test_split(df_dev_test, test_size=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:41:02.230119Z",
     "start_time": "2020-12-11T02:41:02.032170Z"
    }
   },
   "outputs": [],
   "source": [
    "df_train.to_csv(\"./train.csv\", index=False)\n",
    "df_dev.to_csv(\"./dev.csv\", index=False)\n",
    "de_test.to_csv(\"./test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T02:52:57.380093Z",
     "start_time": "2020-12-11T02:52:57.355890Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>label_des</th>\n",
       "      <th>sentence</th>\n",
       "      <th>len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10522</th>\n",
       "      <td>4</td>\n",
       "      <td>同城服务</td>\n",
       "      <td>百万优质货源，一秒找货赚钱。运满满司机端，拥有丰富货源以及百万实名认证货主。让货车司机一键找...</td>\n",
       "      <td>275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10932</th>\n",
       "      <td>71</td>\n",
       "      <td>亲子儿童</td>\n",
       "      <td>宝宝巴士又一童谣精品宝宝童谣之DJ它打破以往轻音乐的形式，以动感的DJ打碟为主，打造全新的摇...</td>\n",
       "      <td>655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6604</th>\n",
       "      <td>46</td>\n",
       "      <td>视频</td>\n",
       "      <td>相声,小品,魔术杂技；评书,笑话,说</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415</th>\n",
       "      <td>100</td>\n",
       "      <td>美颜</td>\n",
       "      <td>手机瞬间变单反相机，一秒拍佳作,,更稳定、更优质，邀您一起体验。</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4746</th>\n",
       "      <td>54</td>\n",
       "      <td>职考</td>\n",
       "      <td>一站式解决执业中药师、执业西药师考试刷题、听课两大需求用看书30的时间顺利高分通过2018年...</td>\n",
       "      <td>472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>7</td>\n",
       "      <td>家政</td>\n",
       "      <td>掌上热力是北京市热力集团有限责任公司面向广大居民热用户推出的掌上互动服务应用软件，旨在为热用...</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9719</th>\n",
       "      <td>70</td>\n",
       "      <td>工具</td>\n",
       "      <td>《激战海湾》是一款海战题材的射击游戏，从上面的预告片可以看到，游戏颇有卡通版《战舰世界》的感...</td>\n",
       "      <td>205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>70</td>\n",
       "      <td>工具</td>\n",
       "      <td>那青春梦象动态壁纸是一款以栅栏，花朵，青春为主要元素的小清新静物类动态壁纸作品，破旧复古的浅...</td>\n",
       "      <td>375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11221</th>\n",
       "      <td>70</td>\n",
       "      <td>工具</td>\n",
       "      <td>性能监视器用于检查设备性能，显示后台应用程序和进程，以及系统中正在运行和已停止的服务。设备性...</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>24</td>\n",
       "      <td>辅助工具</td>\n",
       "      <td>王者代练是陕西绝顶人峰网络科技有限公司开发的一款游戏代练交易管理应用，是国内专业的游戏代练交...</td>\n",
       "      <td>327</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10312 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       label label_des                                           sentence  len\n",
       "10522      4      同城服务  百万优质货源，一秒找货赚钱。运满满司机端，拥有丰富货源以及百万实名认证货主。让货车司机一键找...  275\n",
       "10932     71      亲子儿童  宝宝巴士又一童谣精品宝宝童谣之DJ它打破以往轻音乐的形式，以动感的DJ打碟为主，打造全新的摇...  655\n",
       "6604      46        视频                                 相声,小品,魔术杂技；评书,笑话,说   18\n",
       "415      100        美颜                   手机瞬间变单反相机，一秒拍佳作,,更稳定、更优质，邀您一起体验。   32\n",
       "4746      54        职考  一站式解决执业中药师、执业西药师考试刷题、听课两大需求用看书30的时间顺利高分通过2018年...  472\n",
       "...      ...       ...                                                ...  ...\n",
       "405        7        家政  掌上热力是北京市热力集团有限责任公司面向广大居民热用户推出的掌上互动服务应用软件，旨在为热用...  126\n",
       "9719      70        工具  《激战海湾》是一款海战题材的射击游戏，从上面的预告片可以看到，游戏颇有卡通版《战舰世界》的感...  205\n",
       "616       70        工具  那青春梦象动态壁纸是一款以栅栏，花朵，青春为主要元素的小清新静物类动态壁纸作品，破旧复古的浅...  375\n",
       "11221     70        工具  性能监视器用于检查设备性能，显示后台应用程序和进程，以及系统中正在运行和已停止的服务。设备性...   83\n",
       "38        24      辅助工具  王者代练是陕西绝顶人峰网络科技有限公司开发的一款游戏代练交易管理应用，是国内专业的游戏代练交...  327\n",
       "\n",
       "[10312 rows x 4 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# debug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T06:41:37.254604Z",
     "start_time": "2020-12-11T06:41:37.249096Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T06:42:24.073469Z",
     "start_time": "2020-12-11T06:42:24.058061Z"
    }
   },
   "outputs": [],
   "source": [
    "pres = [14,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        14,\n",
    "        70,\n",
    "        100,\n",
    "        70,\n",
    "        70,\n",
    "        14,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        100,\n",
    "        70,\n",
    "        70,\n",
    "        100,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        14,\n",
    "        100,\n",
    "        70,\n",
    "        100,\n",
    "        100,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        100,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        14,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        100,\n",
    "        70,\n",
    "        70,\n",
    "        70]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T06:42:38.139246Z",
     "start_time": "2020-12-11T06:42:38.123290Z"
    }
   },
   "outputs": [],
   "source": [
    "trues = [17,\n",
    "         59,\n",
    "         92,\n",
    "         53,\n",
    "         96,\n",
    "         53,\n",
    "         34,\n",
    "         38,\n",
    "         17,\n",
    "         58,\n",
    "         70,\n",
    "         111,\n",
    "         40,\n",
    "         24,\n",
    "         36,\n",
    "         71,\n",
    "         94,\n",
    "         70,\n",
    "         21,\n",
    "         9,\n",
    "         113,\n",
    "         70,\n",
    "         17,\n",
    "         49,\n",
    "         26,\n",
    "         45,\n",
    "         71,\n",
    "         83,\n",
    "         17,\n",
    "         46,\n",
    "         14,\n",
    "         53,\n",
    "         34,\n",
    "         70,\n",
    "         54,\n",
    "         18,\n",
    "         94,\n",
    "         71,\n",
    "         88,\n",
    "         53,\n",
    "         84,\n",
    "         106,\n",
    "         71,\n",
    "         47,\n",
    "         39,\n",
    "         24,\n",
    "         98,\n",
    "         85,\n",
    "         34,\n",
    "         28,\n",
    "         66,\n",
    "         117,\n",
    "         11,\n",
    "         22,\n",
    "         83,\n",
    "         71,\n",
    "         25,\n",
    "         104,\n",
    "         78,\n",
    "         20,\n",
    "         102,\n",
    "         25,\n",
    "         34,\n",
    "         114,\n",
    "         70,\n",
    "         4,\n",
    "         90,\n",
    "         95,\n",
    "         17,\n",
    "         94,\n",
    "         70,\n",
    "         18,\n",
    "         41,\n",
    "         118,\n",
    "         16,\n",
    "         70,\n",
    "         19,\n",
    "         114,\n",
    "         14,\n",
    "         71,\n",
    "         24,\n",
    "         75,\n",
    "         16,\n",
    "         53,\n",
    "         85,\n",
    "         70,\n",
    "         95,\n",
    "         34,\n",
    "         96,\n",
    "         27,\n",
    "         17,\n",
    "         96,\n",
    "         49,\n",
    "         25,\n",
    "         36,\n",
    "         70,\n",
    "         53,\n",
    "         70,\n",
    "         1,\n",
    "         5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T07:04:24.113131Z",
     "start_time": "2020-12-11T07:04:24.107923Z"
    }
   },
   "outputs": [],
   "source": [
    "pres = [14,\n",
    "        70,\n",
    "        70,\n",
    "        70,\n",
    "        7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T07:04:24.367084Z",
     "start_time": "2020-12-11T07:04:24.362844Z"
    }
   },
   "outputs": [],
   "source": [
    "trues = [\n",
    "         14,\n",
    "         70,\n",
    "         7,\n",
    "         7,\n",
    "         7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T07:04:24.627429Z",
     "start_time": "2020-12-11T07:04:24.618278Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(trues, pres, average=\"micro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T07:04:24.956664Z",
     "start_time": "2020-12-11T07:04:24.946383Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(trues, pres, average=\"micro\")\n",
    "recall_score(trues, pres, average=\"micro\")\n",
    "precision_score(trues, pres, average=\"micro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-11T07:01:17.800662Z",
     "start_time": "2020-12-11T07:01:17.787254Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.16494845, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        ])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(trues, pres, average=None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
